package cs429_project;

import cs429_project.clustering.DocCluster;
import cs429_project.clustering.DocVec;
import cs429_project.clustering.KMeansClustering;

import java.io.IOException;

import java.math.BigDecimal;

import java.net.MalformedURLException;
import java.net.URL;

import java.util.Iterator;
import java.util.NavigableMap;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;

import org.apache.lucene.queryParser.ParseException;

import org.w3c.dom.Document;

/**
 * Class for storing a query and its corresponding Wikipedia page.  It can also
 * be used to determine whether a query is explicit (EQ), ambiguous (AQ), or
 * broader (BQ).  In addition, there is another classification, when an
 * ambiguous query has been mapped to a non-disambiguation Wikipedia page.
 * Types of query are determined by an {@code enum} located inside of this
 * class, called {@link cs429_project.QueryAndWiki.QueryType}.
 * @author Peter Ferrans
 */
public class QueryAndWiki {

    /** The base URL for all the Wikipedia XML pages.  The specific page name
     * is appended to the end of this to get its URL. */
    public static final String WIKI_BASE_URL = "http://en.wikipedia.org/wiki/"
            + "Special:Export/";
    /** Text of the original query entered by the user. */
    private String queryText;
    /** The name of the Wikipedia page corresponding to the query. */
    private String wikiPageName;
    /** The text of the Wikipedia page corresponding to the query.  If
     * {@code wikiPageName} refers to a page that does not actually exist, this
     * field will be null.*/
    private String wikiPageText;
    /** The type of this query, according to our classification system. */
    private QueryType type;

    /**
     * Creates a new instance for the query specified by the user.
     * @param queryText
     *    Query for the new instance.
     */
    public QueryAndWiki(String queryText) {
        setUp(queryText);
    }

    /**
     * Changes the query for this instance and updates all other fields to
     * correspond with the new query.
     * @param queryText
     *    New query for this instance.
     */
    public void setQueryText(String queryText) {
        setUp(queryText);
    }

    /**
     * Returns the current text of the query in this instance.
     * @return
     *    Current text of query in this instance.
     */
    public String getQueryText() {
        return this.queryText;
    }

    /**
     * Returns the current Wikipedia page name in this instance.
     * @return
     *    Current Wikipedia page name in this instance.
     */
    public String getWikiPageName() {
        return this.wikiPageName;
    }

    /**
     * Returns the current Wikipedia page text for this instance.
     * @return
     *    Current Wikipedia page text for this instance.
     */
    public String getWikiPageText() {
        return this.wikiPageText;
    }

    /**
     * Determines the type (ambiguous, explicit, broader) of this query.
     * @return
     *    The type of this query.
     */
    public QueryType getQueryType() {
        return this.type;
    }

    /**
     * If the query is matched to a disambiguation page, this method determines
     * which link on that page is most likely to be the link corresponding to
     * the entity the user wanted.  First, the initial query is run on the
     * Lucene index being searched, and the top results are returned (the
     * number of results is determined by the parameter {@code maxResults}).
     * K-means clustering is then used to cluster these documents together.
     * The cluster which is most similar to the query is chosen, and its
     * centroid is compared with the text of all the pages linked to on the
     * disambiguation page.  Whichever page is most similar to the centroid is
     * chosen as this query's Wikipedia page, and the query type changes from
     * {@link QueryType#AMBIGUOUS} to {@link QueryType#SOLVED_AMBIGUOUS}.
     * @param indexName
     *    The name of the Lucene index to be searched.
     * @param field
     *    The field of the documents in the Lucene index that we are searching.
     *    This should be the field where the main text is located.
     * @param maxResults
     *    The maximum number of results from the initial query to use in the
     *    clustering evaluation.  A number around 100 is usually good for this.
     * @param k
     *    The number of clusters to be used in k-means clustering.  k should be
     *    less than {@code maxResults}.
     * @throws IOException
     *    Thrown if there is a problem connecting to Wikipedia or accessing the
     *    database.
     * @throws ParseException
     *    Thrown if there is an error in the query when it is being parsed.
     */
    public void solveAmbiguous(String indexName, String field, int maxResults,
            int k) throws IOException, ParseException {
        if (this.isDisambiguationPage()) {
            DocCluster topClust =
                    fetchTopCluster(indexName, field, maxResults, k);
            TreeMap<String, BigDecimal> centroid = topClust.getCurrentMean();
            TreeSet<String> disRefs = WikiPageHandler.getPagesFromWikiText(
                    this.wikiPageText);
            String bestPage = bestPage(centroid, disRefs);

            this.setWikiPageName(makePageName(bestPage));
        }
    }

    /**
     * Sets the mapped Wikipedia page for this query to the page given in the
     * argument.  The text of the page, as well as the type of query, is
     * subsequently updated.  This method is intended mainly to be used with
     * ambiguous queries.  After the Wikipedia page to map to the query has
     * been found, this allows us to set that page.
     * @param pageName
     *    The name of the Wikipedia page.  This name should appear in the
     *    format found in the URL for Wikipedia pages online, as opposed to the
     *    page's display title ("{@code Page_Name}" rather than
     *    "{@code Page Name}", for example).
     */
    protected void setWikiPageName(String pageName) {
        this.wikiPageName = pageName;
        this.extractWikiPageText();
    }

    /**
     * Helper method for {@link QueryAndWiki#solveAmbiguous(java.lang.String,
     * java.lang.String, int, int)}.  All of the parameters are the same as the
     * ones for that method.  It searches the database for the initial query,
     * then clusters the results, and finally returns the top-ranked cluster as
     * compared to the initial query.
     * @param indexName
     * @param field
     * @param maxResults
     * @param k
     * @return
     *    The top-ranked cluster, compared to the initial query.
     * @throws IOException
     * @throws ParseException
     */
    private DocCluster fetchTopCluster(String indexName, String field,
            int maxResults, int k) throws IOException, ParseException {
        TreeSet<DocVec> clusterDocs = DocVec.topQueryResults(this.queryText,
                indexName, field, maxResults);
        KMeansClustering kmc = KMeansClustering.createClusters(k, clusterDocs);
        String[] queryTok = WikiPageHandler.tokenizeText(
                this.queryText).toArray(new String[0]);
        NavigableMap<BigDecimal, DocCluster> clustMap =
                kmc.rankClusters(queryTok);
        DocCluster topClust = clustMap.get(clustMap.firstKey());

        return topClust;
    }

    /**
     * Determines whether a Wikipedia page exists that corresponds to this
     * query.
     * @return
     *    Whether a Wikipedia page exists corresponding to this query.
     */
    private boolean wikiPageExists() {
        return this.wikiPageText != null;
    }

    /**
     * Sets up all member variables for a given query.
     * @param queryText
     *    Text of the query.
     */
    private void setUp(String queryText) {
        this.type = null;
        this.queryText = queryText;
        this.wikiPageName = makePageName(queryText);
        extractWikiPageText();
    }

    /**
     * Sets the type of this query, based on the Wikipedia page found for it.
     * Includes setting the type for ambiguous queries whose Wikipedia pages
     * have been determined.
     */
    private void setQueryType() {
        if (this.type == null) {
            if (wikiPageExists()) {
                if (isDisambiguationPage()) {
                    this.type = QueryType.AMBIGUOUS;
                } else {
                    this.type = QueryType.EXPLICIT;
                }
            } else {
                this.type = QueryType.BROADER;
            }
        } else if ((this.type == QueryType.AMBIGUOUS)
                && (!this.isDisambiguationPage())) {
            this.type = QueryType.SOLVED_AMBIGUOUS;
        }
    }

    /**
     * Extracts the text of this query's Wikipedia page from the web and sets
     * {@code wikiPageText} equal to it.
     */
    private void extractWikiPageText() {
        URL pageURL = makeURL();
        Document doc = WikiPageHandler.getDomDocument(pageURL);
        this.wikiPageText = WikiPageHandler.extractDocumentText(doc);

        if (isRedirectPage()) {
            this.wikiPageName = getRedirectName();
            extractWikiPageText();
        }

        setQueryType();
    }

    /**
     * For a Wikipedia page that is actually a redirect to another page,
     * retrieves the name of the page to which we are being redirected.
     * @return
     *    Name of the page to which we are being redirected.
     */
    private String getRedirectName() {
        int startIndex = this.wikiPageText.indexOf("[[") + 2;
        int endIndex = this.wikiPageText.indexOf("]]");
        String redirName = this.wikiPageText.substring(startIndex, endIndex);
        redirName = redirName.replace(" ", "_");

        return redirName;
    }

    /**
     * From the name of the Wikipedia page for this instance, generates the URL
     * for that page.
     * @return
     *    URL for this instance's Wikipedia page.
     */
    private URL makeURL() {
        URL url = null;
        try {
            String urlStr = WIKI_BASE_URL + this.wikiPageName;
            url = new URL(urlStr);
        } catch (MalformedURLException e) {
            System.err.println(e.getMessage());
        }

        return url;
    }

    /**
     * Indicates whether the current Wikipedia page for this instance is a
     * redirect page.
     * @return
     *    Whether the current Wikipedia page for this instance is a redirect
     *    page.
     */
    private boolean isRedirectPage() {
        boolean result;
        if (this.wikiPageText == null) {
            result = false;
        } else {
            result = this.wikiPageText.contains(WikiMarks.REDIRECT);
        }
        return result;
    }

    /**
     * Indicates whether the current Wikipedia page for this instance is a
     * disambiguation page.
     * @return
     *    Whether the current Wikipedia page for this instance is a
     *    disambiguation page.
     */
    private boolean isDisambiguationPage() {
        boolean result;
        if (this.wikiPageText == null) {
            result = false;
        } else {
            result = WikiMarks.containsDisMark(this.wikiPageText);
        }
        return result;
    }

    /**
     * Makes the name of the Wikipedia page for a query.  If the
     * query reads "{@code This is my query}," the page name will be
     * "{@code This_is_my_query}."
     * @param theQueryText
     *    Text of the query to be converted.
     * @return
     *    The Wikipedia page name for the query.
     */
    private static String makePageName(String theQueryText) {
        String pageName =
                theQueryText.trim().replaceAll(" +", " ").replace(" ", "_");

        return pageName;
    }

    /**
     * Given the name of a Wikipedia page as it appears to the user (without
     * underscores), converts the name to the appropriate URL.
     * @param pageName
     *    Name of the page to be converted.
     * @return
     *    The URL string for the page.
     */
    private static String wikiPageNameToUrl(String pageName) {
        String url = makePageName(pageName);
        url = WIKI_BASE_URL + url;

        return url;
    }

    /**
     * Given a set of Wikipedia page names and a centroid, determines which
     * page has text closest to the centroid, and returns the name of that
     * page.
     * @param centroid
     *    Centroid to which all Wikipedia pages are to be compared.
     * @param pageNames
     *    Names of Wikipedia pages that will be compared to the centroid.
     * @return
     *    Name of the page closest to the centroid.
     * @throws MalformedURLException
     */
    private static String bestPage(TreeMap<String, BigDecimal> centroid,
            Set<String> pageNames) throws MalformedURLException {
        BigDecimal bestSimilarity = BigDecimal.valueOf(-1);
        String bestPage = "";
        Iterator<String> pageIt = pageNames.iterator();

        while (pageIt.hasNext()) {
            String pageName = pageIt.next();
            String pageUrl = wikiPageNameToUrl(pageName);
            String pageText = WikiPageHandler.getPageText(pageUrl);

            if (!(WikiMarks.containsDisMark(pageText))) {
                DocVec docVec =
                        WikiPageHandler.docVecFromWikiPageText(pageText);
                BigDecimal similarity = docVec.cosineSimilarity(centroid);

                if (similarity.compareTo(bestSimilarity) >= 0) {
                    bestSimilarity = similarity;
                    bestPage = pageName;
                }
            }
        }

        return bestPage;
    }

    /**
     * Enumeration to identify the different possible types of queries.
     */
    public enum QueryType {

        /** An explicit query that maps directly to a Wikipedia page. */
        EXPLICIT,
        /** An ambiguous query that maps to a disambiguation Wikipedia page. */
        AMBIGUOUS,
        /** An ambiguous query whose mapping non-disambiguation Wikipedia page
         * has been determined. */
        SOLVED_AMBIGUOUS,
        /** A broader query that does not map to a Wikipedia page. */
        BROADER;
    }
}
